# -*- coding:utf-8 -*-
import requests

from bs4 import BeautifulSoup
import json
import lxml

url = 'https://chs.meituan.com/meishi/'  # 地址
headers = {
    'Host': 'www.meituan.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0',
    'Accept': 'application/json',
    'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
    'Accept-Encoding': 'gzip, deflate, br',
    'Referer': 'https://www.meituan.com/meishi/',
    'Connection': 'keep-alive',
    'Cookie': 'lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; _lxsdk_cuid=16ae54dc806c8-0a9eaf8357db7b-4c312c7c-1fa400-16ae54dc806c8; __mta=19182983.1558625245045.1558625245045.1558625245045.1; ci=70; rvct=70%2C202%2C151; client-id=135f9f58-0248-4852-9052-5e7f7e4128c5; _hc.v=cd2cf02c-88c1-c9fd-3245-79d45534b60f.1558834794; uuid=6133581b5adf4cb9be65.1558871918.1.0.0; _lxsdk_s=16af401c778-cd-27b-dbd%7C%7C27; _lxsdk=16ae54dc806c8-0a9eaf8357db7b-4c312c7c-1fa400-16ae54dc806c8; lat=28.113225; lng=113.009235',
    'Cache-Control': 'max-age=0'
}
# proxy = '180.97.250.130:8562'
proxy = '58.218.205.42:5295'

# 如果代理需要验证，只需要在前面加上用户名密码，如下所示

# proxy='username:password@124.243.226.18:8888'
proxies = {
    'http': 'http://' + proxy,
    'https': 'https://' + proxy
}


def get_start_links(url):
    html = requests.get(url, headers=headers).text
    # print html
    soup = BeautifulSoup(html, 'lxml');
    find_all = soup.find_all("script");
    str="";
    for i in find_all:
        if (len(i) and isinstance(i.contents[0], basestring) and i.contents[0].startswith("window._appState = ")):
            str = i.contents[0].replace("window._appState = ","")[:-1];
            print str;
    get = json.loads(str).get("poiLists").get("totalCounts");
    links = [link.find('div').find("div").find("a")["href"] for link in soup.find_all("ul", class_="list-ul")];
    print links


start_url_list = get_start_links(url)
